From 4f766bc869ba65a012cae8a50a632c3201369548 Mon Sep 17 00:00:00 2001 From: "kaf24@labyrinth.cl.cam.ac.uk" Date: Wed, 25 Aug 2004 16:26:15 +0000 Subject: [PATCH] bitkeeper revision 1.1159.1.105 (412cbda71t8CNbo2C1IX2_T3wi4QIQ) Move count_info outside pfn_info union, to where it belongs. --- xen/arch/x86/domain.c | 16 +++++----- xen/arch/x86/memory.c | 60 +++++++++++++++++++++++------------- xen/common/dom0_ops.c | 2 +- xen/common/dom_mem_ops.c | 6 ++-- xen/common/keyhandler.c | 4 +-- xen/common/page_alloc.c | 14 ++++----- xen/include/asm-x86/mm.h | 33 ++++++++++---------- xen/include/asm-x86/shadow.h | 2 +- 8 files changed, 76 insertions(+), 61 deletions(-) diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 8739506b73..3031d5de69 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -480,7 +480,7 @@ void domain_relinquish_memory(struct domain *d) { page = list_entry(ent, struct pfn_info, list); - if ( test_and_clear_bit(_PGC_allocated, &page->u.inuse.count_info) ) + if ( test_and_clear_bit(_PGC_allocated, &page->count_info) ) put_page(page); } @@ -489,10 +489,10 @@ void domain_relinquish_memory(struct domain *d) { page = list_entry(ent, struct pfn_info, list); - if ( test_and_clear_bit(_PGC_guest_pinned, &page->u.inuse.count_info) ) + if ( test_and_clear_bit(_PGC_guest_pinned, &page->count_info) ) put_page_and_type(page); - if ( test_and_clear_bit(_PGC_allocated, &page->u.inuse.count_info) ) + if ( test_and_clear_bit(_PGC_allocated, &page->count_info) ) put_page(page); /* @@ -668,9 +668,9 @@ int construct_dom0(struct domain *p, mfn++ ) { page = &frame_table[mfn]; - page->u.inuse.domain = p; - page->u.inuse.type_info = 0; - page->u.inuse.count_info = PGC_always_set | PGC_allocated | 1; + page->u.inuse.domain = p; + page->u.inuse.type_info = 0; + page->count_info = PGC_always_set | PGC_allocated | 1; list_add_tail(&page->list, &p->page_list); p->tot_pages++; p->max_pages++; } @@ -715,7 +715,7 @@ int construct_dom0(struct domain *p, *l1tab++ = mk_l1_pgentry((mfn << PAGE_SHIFT) | L1_PROT); page = &frame_table[mfn]; - set_bit(_PGC_tlb_flush_on_type_change, &page->u.inuse.count_info); + set_bit(_PGC_tlb_flush_on_type_change, &page->count_info); if ( !get_page_and_type(page, p, PGT_writable_page) ) BUG(); @@ -739,7 +739,7 @@ int construct_dom0(struct domain *p, /* Get another ref to L2 page so that it can be pinned. */ if ( !get_page_and_type(page, p, PGT_l2_page_table) ) BUG(); - set_bit(_PGC_guest_pinned, &page->u.inuse.count_info); + set_bit(_PGC_guest_pinned, &page->count_info); } else { diff --git a/xen/arch/x86/memory.c b/xen/arch/x86/memory.c index 52c9dcca8d..d2f58b4cfa 100644 --- a/xen/arch/x86/memory.c +++ b/xen/arch/x86/memory.c @@ -146,6 +146,23 @@ void arch_init_memory(void) static void ptwr_disable(void); unsigned long mfn; + /* + * We are rather picky about the layout of 'struct pfn_info'. The + * count_info and domain fields must be adjacent, as we perform atomic + * 64-bit operations on them. Also, just for sanity, we assert the size + * of the structure here. + */ + if ( (offsetof(struct pfn_info, u.inuse.domain) != + (offsetof(struct pfn_info, count_info) + sizeof(u32))) || + (sizeof(struct pfn_info) != 24) ) + { + printk("Weird pfn_info layout (%ld,%ld,%d)\n", + offsetof(struct pfn_info, count_info), + offsetof(struct pfn_info, u.inuse.domain), + sizeof(struct pfn_info)); + for ( ; ; ) ; + } + memset(percpu_info, 0, sizeof(percpu_info)); vm_assist_info[VMASST_TYPE_writable_pagetables].enable = @@ -154,7 +171,7 @@ void arch_init_memory(void) ptwr_disable; for ( mfn = 0; mfn < max_page; mfn++ ) - frame_table[mfn].u.inuse.count_info |= PGC_always_set; + frame_table[mfn].count_info |= PGC_always_set; /* Initialise to a magic of 0x55555555 so easier to spot bugs later. */ memset(machine_to_phys_mapping, 0x55, 4<<20); @@ -182,9 +199,9 @@ void arch_init_memory(void) mfn < virt_to_phys(&machine_to_phys_mapping[1<<20])>>PAGE_SHIFT; mfn++ ) { - frame_table[mfn].u.inuse.count_info |= PGC_allocated | 1; - frame_table[mfn].u.inuse.type_info = PGT_gdt_page | 1; /* non-RW */ - frame_table[mfn].u.inuse.domain = dom_xen; + frame_table[mfn].count_info |= PGC_allocated | 1; + frame_table[mfn].u.inuse.type_info = PGT_gdt_page | 1; /* non-RW */ + frame_table[mfn].u.inuse.domain = dom_xen; } } @@ -417,11 +434,12 @@ get_page_from_l1e( * No need for LOCK prefix -- we know that count_info is never zero * because it contains PGC_always_set. */ + ASSERT(test_bit(_PGC_always_set, &page->count_info)); __asm__ __volatile__( "cmpxchg8b %2" - : "=a" (e), "=d" (count_info), - "=m" (*(volatile u64 *)(&page->u.inuse.domain)) - : "0" (0), "1" (0), "b" (0), "c" (0) ); + : "=d" (e), "=a" (count_info), + "=m" (*(volatile u64 *)(&page->count_info)) + : "0" (0), "1" (0), "c" (0), "b" (0) ); if ( unlikely((count_info & PGC_count_mask) == 0) || unlikely(e == NULL) || unlikely(!get_domain(e)) ) return 0; @@ -434,7 +452,7 @@ get_page_from_l1e( { if ( unlikely(!get_page_type(page, PGT_writable_page)) ) return 0; - set_bit(_PGC_tlb_flush_on_type_change, &page->u.inuse.count_info); + set_bit(_PGC_tlb_flush_on_type_change, &page->count_info); } return 1; @@ -741,7 +759,7 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e) int alloc_page_type(struct pfn_info *page, unsigned int type) { if ( unlikely(test_and_clear_bit(_PGC_tlb_flush_on_type_change, - &page->u.inuse.count_info)) ) + &page->count_info)) ) { struct domain *p = page->u.inuse.domain; if ( unlikely(NEED_FLUSH(tlbflush_time[p->processor], @@ -822,8 +840,8 @@ static int do_extended_command(unsigned long ptr, unsigned long val) break; } - if ( unlikely(test_and_set_bit(_PGC_guest_pinned, - &page->u.inuse.count_info)) ) + if ( unlikely(test_and_set_bit(_PGC_guest_pinned, + &page->count_info)) ) { MEM_LOG("Pfn %08lx already pinned", pfn); put_page_and_type(page); @@ -840,7 +858,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val) ptr, page->u.inuse.domain); } else if ( likely(test_and_clear_bit(_PGC_guest_pinned, - &page->u.inuse.count_info)) ) + &page->count_info)) ) { put_page_and_type(page); put_page(page); @@ -1007,7 +1025,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val) * disappears then the deallocation routine will safely spin. */ nd = page->u.inuse.domain; - y = page->u.inuse.count_info; + y = page->count_info; do { x = y; if ( unlikely((x & (PGC_count_mask|PGC_allocated)) != @@ -1022,9 +1040,9 @@ static int do_extended_command(unsigned long ptr, unsigned long val) } __asm__ __volatile__( LOCK_PREFIX "cmpxchg8b %3" - : "=a" (nd), "=d" (y), "=b" (e), - "=m" (*(volatile u64 *)(&page->u.inuse.domain)) - : "0" (d), "1" (x), "b" (e), "c" (x) ); + : "=d" (nd), "=a" (y), "=c" (e), + "=m" (*(volatile u64 *)(&page->count_info)) + : "0" (d), "1" (x), "c" (e), "b" (x) ); } while ( unlikely(nd != d) || unlikely(y != x) ); @@ -1395,7 +1413,7 @@ void ptwr_reconnect_disconnected(unsigned long addr) l1_pgentry_val(linear_pg_table[(unsigned long)pl2e >> PAGE_SHIFT]) >> PAGE_SHIFT, frame_table[pfn].u.inuse.type_info, - frame_table[pfn].u.inuse.count_info, + frame_table[pfn].count_info, frame_table[pfn].u.inuse.domain->domain)); nl2e = mk_l2_pgentry(l2_pgentry_val(*pl2e) | _PAGE_PRESENT); @@ -1417,7 +1435,7 @@ void ptwr_reconnect_disconnected(unsigned long addr) PTWR_PRINTK(("[A] now pl2e %p l2e %08lx taf %08x/%08x/%u\n", pl2e, l2_pgentry_val(*pl2e), frame_table[pfn].u.inuse.type_info, - frame_table[pfn].u.inuse.count_info, + frame_table[pfn].count_info, frame_table[pfn].u.inuse.domain->domain)); ptwr_info[cpu].disconnected = ENTRIES_PER_L2_PAGETABLE; /* make pt page write protected */ @@ -1535,7 +1553,7 @@ int ptwr_do_page_fault(unsigned long addr) l1_pgentry_t *pl1e; PTWR_PRINTK(("[I] freeing l1 page %p taf %08x/%08x\n", page, page->u.inuse.type_info, - page->u.inuse.count_info)); + page->count_info)); if (ptwr_info[cpu].writable_idx == PTWR_NR_WRITABLES) ptwr_flush_inactive(); ptwr_info[cpu].writables[ptwr_info[cpu].writable_idx] = addr; @@ -1560,7 +1578,7 @@ int ptwr_do_page_fault(unsigned long addr) >> PAGE_SHIFT]) >> PAGE_SHIFT, frame_table[pfn].u.inuse.type_info, - frame_table[pfn].u.inuse.count_info, + frame_table[pfn].count_info, frame_table[pfn].u.inuse.domain->domain)); /* disconnect l1 page */ nl2e = mk_l2_pgentry((l2_pgentry_val(*pl2e) & ~_PAGE_PRESENT)); @@ -1571,7 +1589,7 @@ int ptwr_do_page_fault(unsigned long addr) PTWR_PRINTK(("[A] now pl2e %p l2e %08lx " "taf %08x/%08x/%u\n", pl2e, l2_pgentry_val(*pl2e), frame_table[pfn].u.inuse.type_info, - frame_table[pfn].u.inuse.count_info, + frame_table[pfn].count_info, frame_table[pfn].u.inuse.domain->domain)); ptwr_info[cpu].writable_l1 = addr; pl1e = map_domain_mem(l2_pgentry_to_pagenr(nl2e) << diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c index 1f1e911b73..5ae8924143 100644 --- a/xen/common/dom0_ops.c +++ b/xen/common/dom0_ops.c @@ -628,7 +628,7 @@ long do_dom0_op(dom0_op_t *u_dom0_op) break; } - if ( page->u.inuse.count_info & PGC_guest_pinned ) + if ( page->count_info & PGC_guest_pinned ) type |= LPINTAB; l_arr[j] |= type; put_page(page); diff --git a/xen/common/dom_mem_ops.c b/xen/common/dom_mem_ops.c index 4e9e7425af..ba570d91f7 100644 --- a/xen/common/dom_mem_ops.c +++ b/xen/common/dom_mem_ops.c @@ -82,12 +82,10 @@ static long free_dom_mem(struct domain *d, return i; } - if ( test_and_clear_bit(_PGC_guest_pinned, - &page->u.inuse.count_info) ) + if ( test_and_clear_bit(_PGC_guest_pinned, &page->count_info) ) put_page_and_type(page); - if ( test_and_clear_bit(_PGC_allocated, - &page->u.inuse.count_info) ) + if ( test_and_clear_bit(_PGC_allocated, &page->count_info) ) put_page(page); put_page(page); diff --git a/xen/common/keyhandler.c b/xen/common/keyhandler.c index 6b5872cfbe..16f811d07f 100644 --- a/xen/common/keyhandler.c +++ b/xen/common/keyhandler.c @@ -82,14 +82,14 @@ void do_task_queues(unsigned char key, void *dev_id, { page = list_entry(ent, struct pfn_info, list); printk("Page %08x: caf=%08x, taf=%08x\n", - page_to_phys(page), page->u.inuse.count_info, + page_to_phys(page), page->count_info, page->u.inuse.type_info); } } page = virt_to_page(d->shared_info); printk("Shared_info@%08x: caf=%08x, taf=%08x\n", - page_to_phys(page), page->u.inuse.count_info, + page_to_phys(page), page->count_info, page->u.inuse.type_info); printk("Guest: upcall_pend = %02x, upcall_mask = %02x\n", diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c index 79b8df7452..faa3b6ec3a 100644 --- a/xen/common/page_alloc.c +++ b/xen/common/page_alloc.c @@ -310,9 +310,9 @@ unsigned long alloc_xenheap_pages(int order) for ( i = 0; i < (1 << order); i++ ) { - pg[i].u.inuse.count_info = PGC_always_set; - pg[i].u.inuse.domain = NULL; - pg[i].u.inuse.type_info = 0; + pg[i].count_info = PGC_always_set; + pg[i].u.inuse.domain = NULL; + pg[i].u.inuse.type_info = 0; } return (unsigned long)page_to_virt(pg); @@ -383,9 +383,9 @@ struct pfn_info *alloc_domheap_pages(struct domain *d, int order) } } - pg[i].u.inuse.count_info = PGC_always_set; - pg[i].u.inuse.domain = NULL; - pg[i].u.inuse.type_info = 0; + pg[i].count_info = PGC_always_set; + pg[i].u.inuse.domain = NULL; + pg[i].u.inuse.type_info = 0; } if ( d == NULL ) @@ -411,7 +411,7 @@ struct pfn_info *alloc_domheap_pages(struct domain *d, int order) { pg[i].u.inuse.domain = d; wmb(); /* Domain pointer must be visible before updating refcnt. */ - pg[i].u.inuse.count_info |= PGC_allocated | 1; + pg[i].count_info |= PGC_allocated | 1; list_add_tail(&pg[i].list, &d->page_list); } diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h index c07235fa5e..7bbe5fe06d 100644 --- a/xen/include/asm-x86/mm.h +++ b/xen/include/asm-x86/mm.h @@ -30,25 +30,24 @@ struct pfn_info /* Each frame can be threaded onto a doubly-linked list. */ struct list_head list; + /* Reference count and various PGC_xxx flags and fields. */ + u32 count_info; + /* Context-dependent fields follow... */ union { - /* Page is in use by a domain. */ + /* Page is in use: ((count_info & PGC_count_mask) != 0). */ struct { - /* Owner of this page. */ + /* Owner of this page (NULL if page is anonymous). */ struct domain *domain; - /* Reference count and various PGC_xxx flags and fields. */ - u32 count_info; /* Type reference count and various PGT_xxx flags and fields. */ u32 type_info; } inuse; - /* Page is on a free list. */ + /* Page is on a free list: ((count_info & PGC_count_mask) == 0). */ struct { /* Mask of possibly-tainted TLBs. */ unsigned long cpu_mask; - /* Must be at same offset as 'u.inuse.count_flags'. */ - u32 __unavailable; /* Order-size of the free chunk this page is the head of. */ u8 order; } free; @@ -108,8 +107,8 @@ struct pfn_info wmb(); /* install valid domain ptr before updating refcnt. */ \ spin_lock(&(_dom)->page_alloc_lock); \ /* _dom holds an allocation reference */ \ - ASSERT((_pfn)->u.inuse.count_info == PGC_always_set); \ - (_pfn)->u.inuse.count_info |= PGC_allocated | 1; \ + ASSERT((_pfn)->count_info == PGC_always_set); \ + (_pfn)->count_info |= PGC_allocated | 1; \ if ( unlikely((_dom)->xenheap_pages++ == 0) ) \ get_knownalive_domain(_dom); \ list_add_tail(&(_pfn)->list, &(_dom)->xenpage_list); \ @@ -126,13 +125,13 @@ void free_page_type(struct pfn_info *page, unsigned int type); static inline void put_page(struct pfn_info *page) { - u32 nx, x, y = page->u.inuse.count_info; + u32 nx, x, y = page->count_info; do { x = y; nx = x - 1; } - while ( unlikely((y = cmpxchg(&page->u.inuse.count_info, x, nx)) != x) ); + while ( unlikely((y = cmpxchg(&page->count_info, x, nx)) != x) ); if ( unlikely((nx & PGC_count_mask) == 0) ) free_domheap_page(page); @@ -142,7 +141,7 @@ static inline void put_page(struct pfn_info *page) static inline int get_page(struct pfn_info *page, struct domain *domain) { - u32 x, nx, y = page->u.inuse.count_info; + u32 x, nx, y = page->count_info; struct domain *p, *np = page->u.inuse.domain; do { @@ -160,9 +159,9 @@ static inline int get_page(struct pfn_info *page, } __asm__ __volatile__( LOCK_PREFIX "cmpxchg8b %3" - : "=a" (np), "=d" (y), "=b" (p), - "=m" (*(volatile u64 *)(&page->u.inuse.domain)) - : "0" (p), "1" (x), "b" (p), "c" (nx) ); + : "=d" (np), "=a" (y), "=c" (p), + "=m" (*(volatile u64 *)(&page->count_info)) + : "0" (p), "1" (x), "c" (p), "b" (nx) ); } while ( unlikely(np != p) || unlikely(y != x) ); @@ -254,7 +253,7 @@ static inline int get_page_type(struct pfn_info *page, u32 type) DPRINTK("Error while validating pfn %08lx for type %08x." " caf=%08x taf=%08x\n", page_to_pfn(page), type, - page->u.inuse.count_info, + page->count_info, page->u.inuse.type_info); put_page_type(page); return 0; @@ -292,7 +291,7 @@ static inline int get_page_and_type(struct pfn_info *page, ASSERT(((_p)->u.inuse.type_info & PGT_type_mask) == (_t)); \ ASSERT(((_p)->u.inuse.type_info & PGT_count_mask) != 0) #define ASSERT_PAGE_IS_DOMAIN(_p, _d) \ - ASSERT(((_p)->u.inuse.count_info & PGC_count_mask) != 0); \ + ASSERT(((_p)->count_info & PGC_count_mask) != 0); \ ASSERT((_p)->u.inuse.domain == (_d)) int check_descriptor(unsigned long *d); diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h index cbfca412b7..f874b6be4a 100644 --- a/xen/include/asm-x86/shadow.h +++ b/xen/include/asm-x86/shadow.h @@ -121,7 +121,7 @@ static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn ) mfn, pfn, m->shadow_dirty_bitmap_size, m ); SH_LOG("dom=%u caf=%08x taf=%08x\n", frame_table[mfn].u.inuse.domain->domain, - frame_table[mfn].u.inuse.count_info, + frame_table[mfn].count_info, frame_table[mfn].u.inuse.type_info ); } -- 2.30.2